#coding:utf-8
'''
* coder  : dzlua
* email  : 505544956@qq.com
* module : spider
* path   : module
* file   : spider.py
* time   : 2017-11-29 13:31:56
'''
#--------------------#
import MySQLdb, threadpool, urllib, urllib2
import re, cookielib, codecs, os
#--------------------#

#--------------------#
class Base(object):
    def write(self, text):
        f = codecs.open(
            self.__class__.__name__.lower() + '.log',
            'a', 'utf-8' )
        f.write(text)
        f.close()
    def prt(self, msg):
        errmsg = '[%s] %s' % (
            self.__class__.__name__.lower(),
            msg or '')
        print(errmsg)
    def log(self, msg):
        errmsg = '[%s] %s\n' % (
            self.__class__.__name__.lower(),
            msg or '')
        self.write(errmsg)
    def logp(self, msg):
        errmsg = '[%s] %s' % (
            self.__class__.__name__.lower(),
            msg or '')
        print(errmsg)
        self.write(errmsg + '\n')
    def strip(self, text):
        return re.sub(r'\s+','', text)
    def url_remove_param(self, url):
        return re.sub(r'(\?[a-zA-Z0-9&=]+)?','', url)
    def create_file(self, file, content=u''):
        if os.path.exists(file):
            return False
        path = os.path.dirname(file)
        self.mkdirs(path)
        #
        fp = codecs.open(file, 'w', 'utf-8')
        fp.write(content)
        fp.close()

        return True
    def mkdirs(self, path):
        try:
            os.makedirs(path)
        except:
            pass
    def del_empty_line(self, text):
        return re.sub(r'\n[\s| ]*\n', '', text)
#--------------------#

#--------------------#
class DataBase(Base):
    def __init__(self, host, user, pwd, db, charset='utf8'):
        self.tb = self.__class__.__name__.lower()
        self.db = MySQLdb.connect(
            host=host, user=user,
            passwd=pwd, db=db,
            charset=charset )
        self.prt('open mysql ok. host: %s, user: %s, db: %s' % (
                host, user, db ) )
    def close(self):
        self.db.close()
        self.prt('mysql closed.')
    def execute(self, sql, param=None):
        cursor = self.db.cursor()
        if param:
            cursor.execute(sql, param)
        else:
            cursor.execute(sql)
        cursor.close()
        self.db.commit()
    def select(self, sql, param=None):
        cursor = self.db.cursor()
        if param:
            cursor.execute(sql, param)
        else:
            cursor.execute(sql)
        result = cursor.fetchall()
        cursor.close()
        return result
    def to_binary(self, data):
        return MySQLdb.Binary(data)
#--------------------#

#--------------------#
class SpiderBase(Base):
    def __init__(self, threads=4, tasks=0):
        self.pool = threadpool.ThreadPool(threads, tasks)
        self.headers = {
                'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3012.0 Safari/537.36'
            }
        self.cookie = cookielib.CookieJar()
        self.handler = urllib2.HTTPCookieProcessor(self.cookie)
        self.opener = urllib2.build_opener(self.handler)
    def wait(self):
        self.pool.wait()
    def put(self, callback, url, data = None, params = None, headers=None):
        def getPage(url, params=None, hreaders=None):
            try:
                en_data = None
                if params:
                    en_data = urllib.urlencode(params)
                request = urllib2.Request(
                    url,
                    en_data,
                    hreaders or self.headers )
                response = self.opener.open(request)
                read = response.read()
                return read
            except urllib2.URLError, e:
                if hasattr(e, 'reason'):
                    self.logp(u'Error 连接 %s 失败,错误原因: %s' % (url, e.reason))
                return None
        def fun(url):
            return getPage(url, params, headers)
        def funcallback(a, b):
            callback(self, a.args[0], b, data)
        reqs = threadpool.makeRequests(fun, [url], funcallback)
        [ self.pool.putRequest(req) for req in reqs ]
        self.prt('put url : %s, data : %s' % (url, data))
#--------------------#

#--------------------#
class Spider(DataBase, SpiderBase):
    def __init__(self, host, user, pwd, db, threads=4, tasks=0, charset='utf8'):
        DataBase.__init__(self, host, user, pwd, db, charset)
        SpiderBase.__init__(self, threads, tasks)
    def get_strings(self, soup):
        text = ''
        if not soup:
            return text
        for string in soup.strings:
            text += (string + '\n')
        return text
    def has_data(self, tb, key, value):
        sql = 'SELECT * FROM %s WHERE %s="%s";' % (tb, key, value)
        ret = self.select(sql)
        if not ret or len(ret) == 0:
            return False
        return True
    def url_join(self, host, url):
        if url[0] == '/':
            return os.path.join(host, url[1:])
        else:
            return os.path.join(host, url)
    def url_host(self, url):
        protocol, s1 = urllib.splittype(url)
        host, s2 =  urllib.splithost(s1)
        return '%s://%s' % (protocol, host)
#--------------------#
